package com.chance.cc.crawler.development.bootstrap.zhihu;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.filter.FilterInfo;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import org.apache.commons.lang3.StringUtils;
import org.junit.Test;

import java.security.NoSuchAlgorithmException;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.*;
import static com.chance.cc.crawler.development.scripts.xcar.carseries.forum.XCarSeriesForumCrawlerScript.site;
import static org.codehaus.groovy.runtime.EncodingGroovyMethods.md5;

/**
 * @author bx
 * @date 2021/1/11 0011 11:05
 */
public class ZHSearchStart {

    public static final String domainId = "zhihu";

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }

    public static void main(String[] args) {

       // searchRealTime();
        tst();
        //searchOneHourTime();

    }

    private static void searchRealTime(){
        FilterInfo filterInfo = new FilterInfo();
        filterInfo.setFilter(CrawlerEnum.CrawlerRecordFilter.count);
        filterInfo.setCountTotalNum(100);

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domainId, CrawlerEnum.CrawlerRequestType.turnPage)
                .httpUrl("https://www.zhihu.com/api/v4/search_v3")
                .httpHead("user-agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36")
                .httpHead("cookie","d_c0=\"ADBiHqMywQ6PTlAq-wLKq4redBYLDaR06Hw=|1546263453\"; _xsrf=kYQKJQ39BrawjmneAv07aXGrmZhmHvaz; _ga=GA1.2.1328271267.1582701249; _zap=61c8630f-164c-4912-b2e3-11b8a48cbb83; captcha_session_v2=\"2|1:0|10:1617876364|18:captcha_session_v2|88:bXRFb1FuNmNXTHZKRWl1ZTdoYUpqcVhzRVFRcHpPdTB4WmdJWnpDbFNxeFhmaDBGUFVhMTNQUmxRdFYzamNlVg==|ec41feb8d9cea6edd33b6022246bc5d11fc6848481cb0505a07e28ffde522fa6\"; __snaker__id=1wqY9sxpGpxAdxuU; gdxidpyhxdE=juGf7TvV3b0cN5bbsawbBpMs%5CacehfjL6OK4q9r4Q7RRLjVuOq5%5CrK6cNd0fw78%2FvmDeZOnJ4mwfqm%2B6IZQmqGDJsmW%5CKd5G2NsY5Zqz4wvDDi%2Fd0JGg0SiBjDc4hviDHlQ%2FrkzBYQs0ih32m%2BCyS00yYZUZb8qh%2FX4R8xSjTmbtVvn3%3A1617877266145; _9755xjdesxxd_=32; YD00517437729195%3AWM_TID=i0Pf3X5wrfJERAFVBVM7kI4V7gwbITFX; YD00517437729195%3AWM_NIKE=9ca17ae2e6ffcda170e2e6eea2f463bce7bbb0b865989e8eb7c45a938e9eaaf53c9beb8fd0ea3ba2bc88b2d72af0fea7c3b92a8de7a68aeb52a1aa9a91b880b8aea9d8d4348fee0097ec41bbb2f983bb7bf3b68fb7aa4094efa1cccc4dac8c9c92f24ef79899a9e86f82b68ab9b7339294baaee240edbc9d91f14d9ce9a3bbb57d93eca299c65bb3f5bfd6d352878685a5ed4f9a8faba9db5aa3979d92c167e9ec9695f8688df1a494d86e8d8efd84fc3ab4bc9ea6d037e2a3; YD00517437729195%3AWM_NI=O9jihZcvVp5ttrg7Q8M20viKyn7b8q1jAV%2B4P1o%2BIGhTTqsF5HWHeeBjFX49RLvMc4%2FdSprGX7mZ6wQ4nxGQq%2Bq5PZEZKDnEZQeHXRaISqBm%2B4ZtulO%2FlsUfOR3cy2pdREM%3D; captcha_ticket_v2=\"2|1:0|10:1617876405|17:captcha_ticket_v2|704:eyJ2YWxpZGF0ZSI6IkNOMzFfWjFVVmU0a2YuWFlVQ3dGZzZ3Z2x5UERMOU0ueHoyblpMb3JSbEQ1NXFINkhKSHQwdTJqdWJPa3F4OW1xVnpwSkduWDVEcndWSXlIeHFSLXpNZ2dwQkdtcVlfc0FLVjZTSTRNZzFnQ1FNak1SbzJSZXZhUEhiUEJITHluXy1fNVh2eEVsZWJzY245S3o1STRWTUFvMjYuck91SjlVcENlTjZZUGx3Zmxxd1lIbU1GVXdQVExvLkFOZ2lsa20uV1VxTEp2V2dQMTV5U3luT0xObFFBR0RnR21uRVlwOWhqZFpjdkpqTU1hVk5LU0t2UGNFMVVHNVhNTERtSDFLQVhCRTBITDdaSU10Lm5BQnE2X053ODB6T2syeGVqN0VlZklHZ2FHSzFjelFDZzZjLkhQcEdBLXVVUEZPTHAtS3lDaDBmRFNrRWhSVmRhem91UUN6dHNPZWp0eng1S2FsWFBpRldNN3ZsX2NqbTFScTBCOENBWkJpWENzQkdPRlFSOUdtemhqVU5mbnBxNF9Ub0x1OHdDZ29iZFphbnNOMDlyMTBBSExISVhoamd5bS5TN1UtLXUxLjUxYUItWmZYTW56YmJ4cWlGVl94Q2pNWHZISkNMbTQyaFA4UWQ0N3hVejY3dGFuYXlYOFFxd1licmhpMW5ndHd4NXN1c0RYMyJ9|229a82129a5e888c4a313fc75fffbf2c2949fc0ca954af164dc1d503f872344c\"; z_c0=\"2|1:0|10:1617876406|4:z_c0|92:Mi4xSnpwYklBQUFBQUFBTUdJZW96TEJEaVlBQUFCZ0FsVk50aU5jWVFCV1ZuOHZybWR4a0RFb2c5eVp4dHdfV2p3MG1R|c2947f184094fd480ce7665ec5bcb710603e26c16d1892adea2553b607f9ce0d\"; tst=r; q_c1=8a20d8ca018d48a9be837a64a594eb25|1617877410000|1575450284000; __utma=51854390.1328271267.1582701249.1600809316.1617877411.9; __utmz=51854390.1617877411.9.1.utmcsr=zhihu.com|utmccn=(referral)|utmcmd=referral|utmcct=/question/47696522/answer/1526610644; __utmv=51854390.110--|2=registration_date=20201028=1^3=entry_date=20191204=1; SESSIONID=pssyzSdBpQ0wFcsaUlgKlSNnDkUuMkpODv9e9U0Pf6E; osd=V1sXAUnsTVqRM9m1fOsMSw2MZTBqmj8p6E6M0D6bGmvmdb3AHGrxNfYy3rR8sLV1jCmRt-pb4Wim-HlmEr3WlkA=; JOID=W1wXBULgSlqVONWyfO8HRwqMYTtmnT8t40KL0DqQFmzmcbbMG2r1Pvo13rB3vLJ1iCKdsOpf6mSh-H1tHrrWkks=; Hm_lvt_98beee57fd2ef70ccdd5ca52b9740c49=1617876365,1617878301,1617934556,1617937287; Hm_lpvt_98beee57fd2ef70ccdd5ca52b9740c49=1617943893; KLBRSID=b33d76655747159914ef8c32323d16fd|1617943897|1617943078")
                .releaseTime(System.currentTimeMillis())
                .needWashed(false)
                .needParsed(false)
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .turnPageFilterInfo(filterInfo)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(8*24,null))
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo("filter-zhihu-search-queue"))
                .proxy(proxy)
                .build();

        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);
        requestRecord.tagsCreator().bizTags().addSite("search");

        CrawlerRecord articleFilterCrawlerRecord = new CrawlerRecord();
        articleFilterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange);
        articleFilterCrawlerRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(8*24,null));
        articleFilterCrawlerRecord.addFilterInfo(FilterUtils.memoryFilterKeyInfo("filter-zhihu-search-answer-queue"));
        requestRecord.tagsCreator().bizTags().addCustomKV("article_record_filter_info", JSON.toJSONString(articleFilterCrawlerRecord));

        CrawlerRecord commentFilterCrawlerRecord = new CrawlerRecord();
        commentFilterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.key);
        commentFilterCrawlerRecord.addFilterInfo(FilterUtils.memoryFilterKeyInfo("filter-zhihu-search-comment-queue"));
        requestRecord.tagsCreator().bizTags().addCustomKV("comment_record_filter_info", JSON.toJSONString(commentFilterCrawlerRecord));

        requestRecord.tagsCreator().bizTags().addCustomKV("time_zone", "a_week");

        //初始 comment crawler request record
        String requestQueueName = StringUtils.joinWith("-","crawler",domainId,site,"latest","comment","queue");
        CrawlerRequestRecord initNewsCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("zhihu_news",turnPageItem)
                .httpUrl("http://192.168.1.217:9599/v1/meta/zhihu/keys?site=search")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        DevCrawlerController devCrawlerController = DevCrawlerController.builder()
                .crawlerThreadNum(15)
                .triggerInfo(domainId, domainId, System.currentTimeMillis(), domainId)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(requestQueueName))
                .consoleResultPipeline()
                .fileResultPipeline("F:\\chance_log\\zhihu\\zhihu.json", false)
                .requestRecord(requestRecord)
                .supportRecord(initNewsCrawlerRecord)
                .build("com.chance.cc.crawler.development.scripts.zhihu");
        devCrawlerController.start();
    }

    private static void searchOneHourTime(){
        FilterInfo filterInfo = new FilterInfo();
        filterInfo.setFilter(CrawlerEnum.CrawlerRecordFilter.count);
        filterInfo.setCountTotalNum(100);

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domainId, CrawlerEnum.CrawlerRequestType.turnPage)
                .httpUrl("https://www.zhihu.com/api/v4/search_v3")
                .httpHead("user-agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36")
                .httpHead("cookie","Hm_lpvt_98beee57fd2ef70ccdd5ca52b9740c49=1603961181; capsion_ticket=\"2|1:0|10:1603961137|14:capsion_ticket|44:MGM5NGZlNzZjZTMzNDNjM2EwYmUyMzI4MWI0YjY4ZDQ=|c18722437bdba2ca267627aeb4e469f878a61ce2cc5938edc0f8047e5010f1a0\"; osd=UlEUBkjgr5Pm2SDTZOW1yxoO0MV3jZ7TuOlCjyvbzf2ptRSsOtAiC7zTId9m5eBj69ye-Rx31f6xHb8D0mVMXA0=; JOID=UV0TAErjo5Tg2yPfY-O3yBYJ1sd0gZnVuupOiC3ZzvGusxavNtckCb_fJtlk5uxk7d6d9Rtx1_29GrkB0WlLWg8=; SESSIONID=d9DX3kTK8kfKj7vV7vvgh9BLEWUFAg98pXhsEGpKWai; Hm_lvt_98beee57fd2ef70ccdd5ca52b9740c49=1603961138; d_c0=\"AGAepdT1HBKPTqvLgAk07qzov98-pCmljCg=|1603961136\"; _zap=e0f72354-9e0b-4c44-b304-69034b72eb1c; z_c0=\"2|1:0|10:1603961177|4:z_c0|92:Mi4xRjRaYklBQUFBQUFBWUI2bDFQVWNFaVlBQUFCZ0FsVk5XYy1IWUFCU1dPdmU1SWh2SUFlVjVuMlpVNTJPdDdVajVB|05b5a2e63d1e20ff1caf272792039c50d65ca4d456e285b53713e32238cc812c\"; unlock_ticket=\"AIBa5tqwGxImAAAAYAJVTWGIml_AsMEPydcm9LAvkintpgvU7GzsRw==\"; KLBRSID=4843ceb2c0de43091e0ff7c22eadca8c|1603961180|1603961135; tst=r; _xsrf=b3b9e127-9bde-4d55-ab1b-3e2c2eed5d41")
                .releaseTime(System.currentTimeMillis())
                .needWashed(false)
                .needParsed(false)
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .turnPageFilterInfo(filterInfo)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24,null))
                .proxy(proxy)
                .build();

        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);
        requestRecord.tagsCreator().bizTags().addSite("search");

        CrawlerRecord articleFilterCrawlerRecord = new CrawlerRecord();
        articleFilterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange);
        articleFilterCrawlerRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(24,null));
        requestRecord.tagsCreator().bizTags().addCustomKV("article_record_filter_info", JSON.toJSONString(articleFilterCrawlerRecord));

        CrawlerRecord commentFilterCrawlerRecord = new CrawlerRecord();
        commentFilterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.key);
        commentFilterCrawlerRecord.addFilterInfo(FilterUtils.memoryFilterKeyInfo("filter-zhihu-search-comment-queue"));
        requestRecord.tagsCreator().bizTags().addCustomKV("comment_record_filter_info", JSON.toJSONString(commentFilterCrawlerRecord));

        requestRecord.tagsCreator().bizTags().addCustomKV("time_zone", "a_day");

        //初始 comment crawler request record
        String requestQueueName = StringUtils.joinWith("-","crawler",domainId,site,"latest","comment","queue");
        CrawlerRequestRecord initNewsCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("zhihu_news",turnPageItem)
                .httpUrl("http://192.168.1.217:9599/v1/meta/zhihu/keys?site=search")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        DevCrawlerController devCrawlerController = DevCrawlerController.builder()
                .crawlerThreadNum(15)
                .triggerInfo(domainId, domainId, System.currentTimeMillis(), domainId)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(requestQueueName))
                .consoleResultPipeline()
                .fileResultPipeline("D:\\chance\\log\\zhihu\\zhihu.json", false)
                .requestRecord(requestRecord)
                .supportRecord(initNewsCrawlerRecord)
                .build();
        devCrawlerController.start();
    }
    //https://www.zhihu.com/search?type=content&q=%E6%96%B0%E7%96%86%E6%A3%89%E8%8A%B1


    public static void tst(){

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domainId, CrawlerEnum.CrawlerRequestType.turnPage)
                .httpUrl("https://www.zhihu.com/search?type=content&q=%E6%96%B0%E7%96%86%E6%A3%89%E8%8A%B1")
                .httpHead("user-agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36")
                .httpHead("cookie","Hm_lpvt_98beee57fd2ef70ccdd5ca52b9740c49=1603961181; capsion_ticket=\"2|1:0|10:1603961137|14:capsion_ticket|44:MGM5NGZlNzZjZTMzNDNjM2EwYmUyMzI4MWI0YjY4ZDQ=|c18722437bdba2ca267627aeb4e469f878a61ce2cc5938edc0f8047e5010f1a0\"; osd=UlEUBkjgr5Pm2SDTZOW1yxoO0MV3jZ7TuOlCjyvbzf2ptRSsOtAiC7zTId9m5eBj69ye-Rx31f6xHb8D0mVMXA0=; JOID=UV0TAErjo5Tg2yPfY-O3yBYJ1sd0gZnVuupOiC3ZzvGusxavNtckCb_fJtlk5uxk7d6d9Rtx1_29GrkB0WlLWg8=; SESSIONID=d9DX3kTK8kfKj7vV7vvgh9BLEWUFAg98pXhsEGpKWai; Hm_lvt_98beee57fd2ef70ccdd5ca52b9740c49=1603961138; d_c0=\"AGAepdT1HBKPTqvLgAk07qzov98-pCmljCg=|1603961136\"; _zap=e0f72354-9e0b-4c44-b304-69034b72eb1c; z_c0=\"2|1:0|10:1603961177|4:z_c0|92:Mi4xRjRaYklBQUFBQUFBWUI2bDFQVWNFaVlBQUFCZ0FsVk5XYy1IWUFCU1dPdmU1SWh2SUFlVjVuMlpVNTJPdDdVajVB|05b5a2e63d1e20ff1caf272792039c50d65ca4d456e285b53713e32238cc812c\"; unlock_ticket=\"AIBa5tqwGxImAAAAYAJVTWGIml_AsMEPydcm9LAvkintpgvU7GzsRw==\"; KLBRSID=4843ceb2c0de43091e0ff7c22eadca8c|1603961180|1603961135; tst=r; _xsrf=b3b9e127-9bde-4d55-ab1b-3e2c2eed5d41")
                .releaseTime(System.currentTimeMillis())
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24,null))
                .proxy(proxy)
                .build();

        requestRecord.tagsCreator().bizTags().addSite("search");

        CrawlerRecord articleFilterCrawlerRecord = new CrawlerRecord();
        articleFilterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange);
        articleFilterCrawlerRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(24,null));
        requestRecord.tagsCreator().bizTags().addCustomKV("article_record_filter_info", JSON.toJSONString(articleFilterCrawlerRecord));

        CrawlerRecord commentFilterCrawlerRecord = new CrawlerRecord();
        commentFilterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.key);
        commentFilterCrawlerRecord.addFilterInfo(FilterUtils.memoryFilterKeyInfo("filter-zhihu-search-comment-queue"));
        requestRecord.tagsCreator().bizTags().addCustomKV("comment_record_filter_info", JSON.toJSONString(commentFilterCrawlerRecord));

        requestRecord.tagsCreator().bizTags().addCustomKV("time_zone", "a_day");


        DevCrawlerController devCrawlerController = DevCrawlerController.builder()
                .crawlerThreadNum(1)
                .triggerInfo(domainId, domainId, System.currentTimeMillis(), domainId)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domainId))
                .consoleResultPipeline()
                .requestRecord(requestRecord)
                .build("com.chance.cc.crawler.development.scripts.zhihu");
        devCrawlerController.start();



    }


    @Test
    public void test() throws NoSuchAlgorithmException {
        String s = md5("3_2.0+/api/v4/search_v3?t=general&q=nike&correction=1&offset=0&limit=20&lc_idx=0&show_all_topics=0&time_zone=a_day+\"ADBdXIpwHhOPTgWa0UcVkbYiDq9T02HXQ4I=|1621240282\"".getBytes());
        System.out.println(s);

    }
}
